rm(list=ls())

# This script takes the count data and adds in the lagged variables and other variables to make the final analysis datasets.

require(tidyverse)
require(lubridate)
files = list.files('data/counts')
files = files[!grepl('fl09',files)]
load("data/misc/USA.habit.rdata")



# Use Coppock and Green replication data to match birthday cohort lagged variables
USA$date_yearlag = ymd(USA$date_yearlag)
USA$birthdate = ymd(USA$birthdate)


  
for(file in files){
  load(paste0('data/counts/',file))
  new_dd$date_year_lag = NA

  
  new_dd$mday = mday(as.Date(new_dd$dob))
  new_dd=new_dd[!(new_dd$year %in% seq(1960, 2000, by = 4) & new_dd$month == "february" & new_dd$mday == 29),] 

new_dd$lag2016 = NA
new_dd$lag2016_white = NA
new_dd$lag2016_black = NA
new_dd$lag2016_hispanic = NA
new_dd$lag2016_female_white = NA
new_dd$lag2016_female_black = NA
new_dd$lag2016_female_hispanic = NA
new_dd$lag2016_male_white = NA
new_dd$lag2016_male_black = NA
new_dd$lag2016_male_hispanic = NA

new_dd$lag2014 = NA
new_dd$lag2014_white = NA
new_dd$lag2014_black = NA
new_dd$lag2014_hispanic = NA
new_dd$lag2014_female_white = NA
new_dd$lag2014_female_black = NA
new_dd$lag2014_female_hispanic = NA
new_dd$lag2014_male_white = NA
new_dd$lag2014_male_black = NA
new_dd$lag2014_male_hispanic = NA

new_dd$lag2012 = NA
new_dd$lag2012_white = NA
new_dd$lag2012_black = NA
new_dd$lag2012_hispanic = NA
new_dd$lag2012_female_white = NA
new_dd$lag2012_female_black = NA
new_dd$lag2012_female_hispanic = NA
new_dd$lag2012_male_white = NA
new_dd$lag2012_male_black = NA
new_dd$lag2012_male_hispanic = NA

new_dd$lag2010 = NA
new_dd$lag2010_white = NA
new_dd$lag2010_black = NA
new_dd$lag2010_hispanic = NA
new_dd$lag2010_female_white = NA
new_dd$lag2010_female_black = NA
new_dd$lag2010_female_hispanic = NA
new_dd$lag2010_male_white = NA
new_dd$lag2010_male_black = NA
new_dd$lag2010_male_hispanic = NA

new_dd$lag2008 = NA
new_dd$lag2008_white = NA
new_dd$lag2008_black = NA
new_dd$lag2008_hispanic = NA
new_dd$lag2008_female_white = NA
new_dd$lag2008_female_black = NA
new_dd$lag2008_female_hispanic = NA
new_dd$lag2008_male_white = NA
new_dd$lag2008_male_black = NA
new_dd$lag2008_male_hispanic = NA

new_dd$lag2006 = NA
new_dd$lag2006_white = NA
new_dd$lag2006_black = NA
new_dd$lag2006_hispanic = NA
new_dd$lag2006_female_white = NA
new_dd$lag2006_female_black = NA
new_dd$lag2006_female_hispanic = NA
new_dd$lag2006_male_white = NA
new_dd$lag2006_male_black = NA
new_dd$lag2006_male_hispanic = NA

new_dd$lag2004 = NA
new_dd$lag2004_white = NA
new_dd$lag2004_black = NA
new_dd$lag2004_hispanic = NA
new_dd$lag2004_female_white = NA
new_dd$lag2004_female_black = NA
new_dd$lag2004_female_hispanic = NA
new_dd$lag2004_male_white = NA
new_dd$lag2004_male_black = NA
new_dd$lag2004_male_hispanic = NA


new_dd$lag2002 = NA
new_dd$lag2002_white = NA
new_dd$lag2002_black = NA
new_dd$lag2002_hispanic = NA
new_dd$lag2002_female_white = NA
new_dd$lag2002_female_black = NA
new_dd$lag2002_female_hispanic = NA
new_dd$lag2002_male_white = NA
new_dd$lag2002_male_black = NA
new_dd$lag2002_male_hispanic = NA

new_dd$lag2000 = NA
new_dd$lag2000_white = NA
new_dd$lag2000_black = NA
new_dd$lag2000_hispanic = NA
new_dd$lag2000_female_white = NA
new_dd$lag2000_female_black = NA
new_dd$lag2000_female_hispanic = NA
new_dd$lag2000_male_white = NA
new_dd$lag2000_male_black = NA
new_dd$lag2000_male_hispanic = NA

new_dd$lagdem = NA
new_dd$lagdem_white = NA
new_dd$lagdem_black = NA
new_dd$lagdem_hispanic = NA
new_dd$lagdem_female_white = NA
new_dd$lagdem_female_black = NA
new_dd$lagdem_female_hispanic = NA
new_dd$lagdem_male_white = NA
new_dd$lagdem_male_black = NA
new_dd$lagdem_male_hispanic = NA

new_dd$lagrep = NA
new_dd$lagrep_white = NA
new_dd$lagrep_black = NA
new_dd$lagrep_hispanic = NA
new_dd$lagrep_female_white = NA
new_dd$lagrep_female_black = NA
new_dd$lagrep_female_hispanic = NA
new_dd$lagrep_male_white = NA
new_dd$lagrep_male_black = NA
new_dd$lagrep_male_hispanic = NA

new_dd$lagregistered = NA
new_dd$lagregistered_white = NA
new_dd$lagregistered_black = NA
new_dd$lagregistered_hispanic = NA
new_dd$lagregistered_female_white = NA
new_dd$lagregistered_female_black = NA
new_dd$lagregistered_female_hispanic = NA
new_dd$lagregistered_male_white = NA
new_dd$lagregistered_male_black = NA
new_dd$lagregistered_male_hispanic = NA


for(i in 1:nrow(new_dd)){
  y = USA$date_yearlag[which(USA$birthdate==new_dd$dob[i])][1]
  if(length(y > 0)){
    new_dd$date_year_lag[i] = y
  }
  x =   which(new_dd$dob == new_dd$date_year_lag[i])
  if(length(x) > 0){
    
  
    new_dd$lag2016[i] = new_dd$vote.gen2016[x]
    new_dd$lag2016_white[i] = new_dd$vote.gen2016_white[x]
  new_dd$lag2016_black[i] = new_dd$vote.gen2016_black[x]
  new_dd$lag2016_hispanic[i] = new_dd$vote.gen2016_hispanic[x]
  new_dd$lag2016_female_white[i] = new_dd$vote.gen2016_female_white[x]
  new_dd$lag2016_female_black[i] = new_dd$vote.gen2016_female_black[x]
  new_dd$lag2016_female_hispanic[i] = new_dd$vote.gen2016_female_hispanic[x]
  new_dd$lag2016_male_white[i] = new_dd$vote.gen2016_male_white[x]
  new_dd$lag2016_male_black[i] = new_dd$vote.gen2016_male_black[x]
  new_dd$lag2016_male_hispanic[i] = new_dd$vote.gen2016_male_hispanic[x]
  
  new_dd$lag2014[i] = new_dd$vote.gen2014[x]
  new_dd$lag2014_white[i] = new_dd$vote.gen2014_white[x]
  new_dd$lag2014_black[i] = new_dd$vote.gen2014_black[x]
  new_dd$lag2014_hispanic[i] = new_dd$vote.gen2014_hispanic[x]
  new_dd$lag2014_female_white[i] = new_dd$vote.gen2014_female_white[x]
  new_dd$lag2014_female_black[i] = new_dd$vote.gen2014_female_black[x]
  new_dd$lag2014_female_hispanic[i] = new_dd$vote.gen2014_female_hispanic[x]
  new_dd$lag2014_male_white[i] = new_dd$vote.gen2014_male_white[x]
  new_dd$lag2014_male_black[i] = new_dd$vote.gen2014_male_black[x]
  new_dd$lag2014_male_hispanic[i] = new_dd$vote.gen2014_male_hispanic[x]
  
  
  new_dd$lag2012[i] = new_dd$vote.gen2012[x]
  new_dd$lag2012_white[i] = new_dd$vote.gen2012_white[x]
  new_dd$lag2012_black[i] = new_dd$vote.gen2012_black[x]
  new_dd$lag2012_hispanic[i] = new_dd$vote.gen2012_hispanic[x]
  new_dd$lag2012_female_white[i] = new_dd$vote.gen2012_female_white[x]
  new_dd$lag2012_female_black[i] = new_dd$vote.gen2012_female_black[x]
  new_dd$lag2012_female_hispanic[i] = new_dd$vote.gen2012_female_hispanic[x]
  new_dd$lag2012_male_white[i] = new_dd$vote.gen2012_male_white[x]
  new_dd$lag2012_male_black[i] = new_dd$vote.gen2012_male_black[x]
  new_dd$lag2012_male_hispanic[i] = new_dd$vote.gen2012_male_hispanic[x]
  
  new_dd$lag2010[i] = new_dd$vote.gen2010[x]
  new_dd$lag2010_white[i] = new_dd$vote.gen2010_white[x]
  new_dd$lag2010_black[i] = new_dd$vote.gen2010_black[x]
  new_dd$lag2010_hispanic[i] = new_dd$vote.gen2010_hispanic[x]
  new_dd$lag2010_female_white[i] = new_dd$vote.gen2010_female_white[x]
  new_dd$lag2010_female_black[i] = new_dd$vote.gen2010_female_black[x]
  new_dd$lag2010_female_hispanic[i] = new_dd$vote.gen2010_female_hispanic[x]
  new_dd$lag2010_male_white[i] = new_dd$vote.gen2010_male_white[x]
  new_dd$lag2010_male_black[i] = new_dd$vote.gen2010_male_black[x]
  new_dd$lag2010_male_hispanic[i] = new_dd$vote.gen2010_male_hispanic[x]
  
  new_dd$lag2008[i] = new_dd$vote.gen2008[x]
  new_dd$lag2008_white[i] = new_dd$vote.gen2008_white[x]
  new_dd$lag2008_black[i] = new_dd$vote.gen2008_black[x]
  new_dd$lag2008_hispanic[i] = new_dd$vote.gen2008_hispanic[x]
  new_dd$lag2008_female_white[i] = new_dd$vote.gen2008_female_white[x]
  new_dd$lag2008_female_black[i] = new_dd$vote.gen2008_female_black[x]
  new_dd$lag2008_female_hispanic[i] = new_dd$vote.gen2008_female_hispanic[x]
  new_dd$lag2008_male_white[i] = new_dd$vote.gen2008_male_white[x]
  new_dd$lag2008_male_black[i] = new_dd$vote.gen2008_male_black[x]
  new_dd$lag2008_male_hispanic[i] = new_dd$vote.gen2008_male_hispanic[x]
  
  new_dd$lag2006[i] = new_dd$vote.gen2006[x]
  new_dd$lag2006_white[i] = new_dd$vote.gen2006_white[x]
  new_dd$lag2006_black[i] = new_dd$vote.gen2006_black[x]
  new_dd$lag2006_hispanic[i] = new_dd$vote.gen2006_hispanic[x]
  new_dd$lag2006_female_white[i] = new_dd$vote.gen2006_female_white[x]
  new_dd$lag2006_female_black[i] = new_dd$vote.gen2006_female_black[x]
  new_dd$lag2006_female_hispanic[i] = new_dd$vote.gen2006_female_hispanic[x]
  new_dd$lag2006_male_white[i] = new_dd$vote.gen2006_male_white[x]
  new_dd$lag2006_male_black[i] = new_dd$vote.gen2006_male_black[x]
  new_dd$lag2006_male_hispanic[i] = new_dd$vote.gen2006_male_hispanic[x]
  
  new_dd$lag2004[i] = new_dd$vote.gen2004[x]
  new_dd$lag2004_white[i] = new_dd$vote.gen2004_white[x]
  new_dd$lag2004_black[i] = new_dd$vote.gen2004_black[x]
  new_dd$lag2004_hispanic[i] = new_dd$vote.gen2004_hispanic[x]
  new_dd$lag2004_female_white[i] = new_dd$vote.gen2004_female_white[x]
  new_dd$lag2004_female_black[i] = new_dd$vote.gen2004_female_black[x]
  new_dd$lag2004_female_hispanic[i] = new_dd$vote.gen2004_female_hispanic[x]
  new_dd$lag2004_male_white[i] = new_dd$vote.gen2004_male_white[x]
  new_dd$lag2004_male_black[i] = new_dd$vote.gen2004_male_black[x]
  new_dd$lag2004_male_hispanic[i] = new_dd$vote.gen2004_male_hispanic[x]
  
  new_dd$lag2002[i] = new_dd$vote.gen2002[x]
  new_dd$lag2002_white[i] = new_dd$vote.gen2002_white[x]
  new_dd$lag2002_black[i] = new_dd$vote.gen2002_black[x]
  new_dd$lag2002_hispanic[i] = new_dd$vote.gen2002_hispanic[x]
  new_dd$lag2002_female_white[i] = new_dd$vote.gen2002_female_white[x]
  new_dd$lag2002_female_black[i] = new_dd$vote.gen2002_female_black[x]
  new_dd$lag2002_female_hispanic[i] = new_dd$vote.gen2002_female_hispanic[x]
  new_dd$lag2002_male_white[i] = new_dd$vote.gen2002_male_white[x]
  new_dd$lag2002_male_black[i] = new_dd$vote.gen2002_male_black[x]
  new_dd$lag2002_male_hispanic[i] = new_dd$vote.gen2002_male_hispanic[x]
  
  new_dd$lag2000[i] = new_dd$vote.gen2000[x]
  new_dd$lag2000_white[i] = new_dd$vote.gen2000_white[x]
  new_dd$lag2000_black[i] = new_dd$vote.gen2000_black[x]
  new_dd$lag2000_hispanic[i] = new_dd$vote.gen2000_hispanic[x]
  new_dd$lag2000_female_white[i] = new_dd$vote.gen2000_female_white[x]
  new_dd$lag2000_female_black[i] = new_dd$vote.gen2000_female_black[x]
  new_dd$lag2000_female_hispanic[i] = new_dd$vote.gen2000_female_hispanic[x]
  new_dd$lag2000_male_white[i] = new_dd$vote.gen2000_male_white[x]
  new_dd$lag2000_male_black[i] = new_dd$vote.gen2000_male_black[x]
  new_dd$lag2000_male_hispanic[i] = new_dd$vote.gen2000_male_hispanic[x]
  
  new_dd$lagdem[i] = new_dd$democrat[x]
  new_dd$lagdem_white[i] = new_dd$democrat_white[x]
  new_dd$lagdem_black[i] = new_dd$democrat_black[x]
  new_dd$lagdem_hispanic[i] = new_dd$democrat_hispanic[x]
  new_dd$lagdem_female_white[i] = new_dd$democrat_female_white[x]
  new_dd$lagdem_female_black[i] = new_dd$democrat_female_black[x]
  new_dd$lagdem_female_hispanic[i] = new_dd$democrat_female_hispanic[x]
  new_dd$lagdem_male_white[i] = new_dd$democrat_male_white[x]
  new_dd$lagdem_male_black[i] = new_dd$democrat_male_black[x]
  new_dd$lagdem_male_hispanic[i] = new_dd$democrat_male_hispanic[x]
  
  new_dd$lagrep[i] = new_dd$republican[x]
  new_dd$lagrep_white[i] = new_dd$republican_white[x]
  new_dd$lagrep_black[i] = new_dd$republican_black[x]
  new_dd$lagrep_hispanic[i] = new_dd$republican_hispanic[x]
  new_dd$lagrep_female_white[i] = new_dd$republican_female_white[x]
  new_dd$lagrep_female_black[i] = new_dd$republican_female_black[x]
  new_dd$lagrep_female_hispanic[i] = new_dd$republican_female_hispanic[x]
  new_dd$lagrep_male_white[i] = new_dd$republican_male_white[x]
  new_dd$lagrep_male_black[i] = new_dd$republican_male_black[x]
  new_dd$lagrep_male_hispanic[i] = new_dd$republican_male_hispanic[x]
  

  new_dd$lagregistered[i] = new_dd$registered[x]
  new_dd$lagregistered_white[i] = new_dd$white[x]
  new_dd$lagregistered_black[i] = new_dd$black[x]
  new_dd$lagregistered_hispanic[i] = new_dd$hispanic[x]
  new_dd$lagregistered_female_white[i] = new_dd$registered_female_white[x]
  new_dd$lagregistered_female_black[i] = new_dd$registered_female_black[x]
  new_dd$lagregistered_female_hispanic[i] = new_dd$registered_female_hispanic[x]
  new_dd$lagregistered_male_white[i] = new_dd$registered_male_white[x]
  new_dd$lagregistered_male_black[i] = new_dd$registered_male_black[x]
  new_dd$lagregistered_male_hispanic[i] = new_dd$registered_male_hispanic[x]
  


  }
}

data = new_dd
data =data[data$year %in% 1968:1995 & !is.na(data$dob) ,]
st=substr(file, 8,9)
filename=paste0("rdd_", st,'.Rdata')
save(data, file = paste0("data/analysis/",filename))
rm(data, new_dd)
print(paste(which(files==file), "states down,", (25-which(files==file)), "states to go!"))
}